In [1]:

    

from keras.models import model_from_yaml
import  numpy as np


    

    




Using gpu device 0: GeForce GTX 960

In [120]:

    

from vectorizer import Vectorizer
py="\n\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, JZS1, GRU\n" * 20


    

In [121]:

    

vec = Vectorizer(py, cutoff=60)

print(dict(enumerate(vec.dictionary)))


    

    




{0: '', 1: 'r', 2: 'e', 3: ' ', 4: 'm', 5: 'o', 6: 'a', 7: 's', 8: 't', 9: '\n', 10: 'S', 11: 'i', 12: 'l', 13: '.', 14: 'k', 15: 'u', 16: ',', 17: 'f', 18: 'p', 19: 'n', 20: '1', 21: 'J', 22: 'G', 23: 'Z', 24: 'c', 25: 'T', 26: 'U', 27: 'y', 28: 'd', 29: 'q', 30: 'L', 31: 'M', 32: 'R'}

In [123]:

    

window_len = 40
matrix = vec.to_matrix(py)
trainX = []
trainy = []

for i in range(len(matrix)-window_len-1):
    trainX.append(matrix[i:i+window_len])
    trainy.append(matrix[i+window_len]) 

trainX = np.array(trainX,dtype="float32")
trainy = np.array(trainy,dtype="float32")

print(trainX.shape)
print(trainy.shape)


    

    




(1719, 40, 33)
(1719, 33)

In [141]:

    

from keras.models import Sequential
from keras.optimizers import Adadelta, SGD, Adagrad, RMSprop
from keras.layers.core import Dense, TimeDistributedDense, Flatten
from keras.layers.recurrent import LSTM, JZS1, GRU

lstmmodel = Sequential()
lstmmodel.add(LSTM(33,60, return_sequences=True))
lstmmodel.add(LSTM(60,60))
lstmmodel.add(Dense(60,33))
lstmmodel.compile(optimizer=SGD(lr=0.1,momentum=0.8),loss='mse',class_mode="categorical")

print("Initial loss")
print(lstmmodel.evaluate(trainX,trainy,show_accuracy=True))
print("Training")
lstmmodel.fit(trainX,trainy,nb_epoch = 100, show_accuracy=True,verbose=1)
print("New loss")
print(lstmmodel.evaluate(trainX,trainy,show_accuracy=True))


    

    




Initial loss
1719/1719 [==============================] - 1s     
[0.044052711637596394, 0.0]
Training
Epoch 0
1719/1719 [==============================] - 6s - loss: 0.0355 - acc: 0.0355     
Epoch 1
1719/1719 [==============================] - 6s - loss: 0.0310 - acc: 0.0913     
Epoch 2
1719/1719 [==============================] - 6s - loss: 0.0300 - acc: 0.1070     
Epoch 3
1719/1719 [==============================] - 6s - loss: 0.0295 - acc: 0.1193     
Epoch 4
1719/1719 [==============================] - 6s - loss: 0.0293 - acc: 0.1280     
Epoch 5
1719/1719 [==============================] - 6s - loss: 0.0290 - acc: 0.1326     
Epoch 6
1719/1719 [==============================] - 6s - loss: 0.0288 - acc: 0.1344     
Epoch 7
1719/1719 [==============================] - 6s - loss: 0.0287 - acc: 0.1373     
Epoch 8
1719/1719 [==============================] - 6s - loss: 0.0285 - acc: 0.1559     
Epoch 9
1719/1719 [==============================] - 6s - loss: 0.0284 - acc: 0.1542     
Epoch 10
1719/1719 [==============================] - 6s - loss: 0.0283 - acc: 0.1798     
Epoch 11
1719/1719 [==============================] - 6s - loss: 0.0282 - acc: 0.1803     
Epoch 12
1719/1719 [==============================] - 6s - loss: 0.0281 - acc: 0.1838     
Epoch 13
1719/1719 [==============================] - 6s - loss: 0.0280 - acc: 0.1879     
Epoch 14
1719/1719 [==============================] - 6s - loss: 0.0279 - acc: 0.1966     
Epoch 15
1719/1719 [==============================] - 6s - loss: 0.0279 - acc: 0.2001     
Epoch 16
1719/1719 [==============================] - 6s - loss: 0.0278 - acc: 0.2024     
Epoch 17
1719/1719 [==============================] - 6s - loss: 0.0277 - acc: 0.2065     
Epoch 18
1719/1719 [==============================] - 6s - loss: 0.0276 - acc: 0.2141     
Epoch 19
1719/1719 [==============================] - 6s - loss: 0.0276 - acc: 0.2135     
Epoch 20
1719/1719 [==============================] - 6s - loss: 0.0275 - acc: 0.2129     
Epoch 21
1719/1719 [==============================] - 6s - loss: 0.0274 - acc: 0.2426     
Epoch 22
1719/1719 [==============================] - 6s - loss: 0.0273 - acc: 0.2333     
Epoch 23
1719/1719 [==============================] - 6s - loss: 0.0273 - acc: 0.2275     
Epoch 24
1719/1719 [==============================] - 6s - loss: 0.0272 - acc: 0.2455     
Epoch 25
1719/1719 [==============================] - 6s - loss: 0.0272 - acc: 0.2455     
Epoch 26
1719/1719 [==============================] - 6s - loss: 0.0271 - acc: 0.2577     
Epoch 27
1719/1719 [==============================] - 6s - loss: 0.0271 - acc: 0.2600     
Epoch 28
1719/1719 [==============================] - 6s - loss: 0.0270 - acc: 0.2624     
Epoch 29
1719/1719 [==============================] - 6s - loss: 0.0269 - acc: 0.2711     
Epoch 30
1719/1719 [==============================] - 6s - loss: 0.0269 - acc: 0.2781     
Epoch 31
1719/1719 [==============================] - 6s - loss: 0.0268 - acc: 0.2740     
Epoch 32
1719/1719 [==============================] - 7s - loss: 0.0267 - acc: 0.2984     
Epoch 33
1719/1719 [==============================] - 7s - loss: 0.0267 - acc: 0.2827     
Epoch 34
1719/1719 [==============================] - 6s - loss: 0.0267 - acc: 0.2868     
Epoch 35
1719/1719 [==============================] - 6s - loss: 0.0266 - acc: 0.2944     
Epoch 36
1719/1719 [==============================] - 6s - loss: 0.0265 - acc: 0.3147     
Epoch 37
1719/1719 [==============================] - 6s - loss: 0.0265 - acc: 0.2827     
Epoch 38
1719/1719 [==============================] - 6s - loss: 0.0264 - acc: 0.3077     
Epoch 39
1719/1719 [==============================] - 6s - loss: 0.0264 - acc: 0.3316     
Epoch 40
1719/1719 [==============================] - 6s - loss: 0.0263 - acc: 0.3159     
Epoch 41
1719/1719 [==============================] - 6s - loss: 0.0264 - acc: 0.3106     
Epoch 42
1719/1719 [==============================] - 6s - loss: 0.0263 - acc: 0.3089     
Epoch 43
1719/1719 [==============================] - 6s - loss: 0.0262 - acc: 0.3293     
Epoch 44
1719/1719 [==============================] - 6s - loss: 0.0261 - acc: 0.3165     
Epoch 45
1719/1719 [==============================] - 6s - loss: 0.0261 - acc: 0.3269     
Epoch 46
1719/1719 [==============================] - 6s - loss: 0.0260 - acc: 0.3165     
Epoch 47
1719/1719 [==============================] - 6s - loss: 0.0259 - acc: 0.3194     
Epoch 48
1719/1719 [==============================] - 6s - loss: 0.0259 - acc: 0.3351     
Epoch 49
1719/1719 [==============================] - 6s - loss: 0.0258 - acc: 0.3403     
Epoch 50
1719/1719 [==============================] - 6s - loss: 0.0258 - acc: 0.3362     
Epoch 51
1719/1719 [==============================] - 6s - loss: 0.0257 - acc: 0.3450     
Epoch 52
1719/1719 [==============================] - 6s - loss: 0.0258 - acc: 0.3223     
Epoch 53
1719/1719 [==============================] - 6s - loss: 0.0257 - acc: 0.3421     
Epoch 54
1719/1719 [==============================] - 6s - loss: 0.0256 - acc: 0.3514     
Epoch 55
1719/1719 [==============================] - 6s - loss: 0.0256 - acc: 0.3613     
Epoch 56
1719/1719 [==============================] - 6s - loss: 0.0255 - acc: 0.3432     
Epoch 57
1719/1719 [==============================] - 6s - loss: 0.0254 - acc: 0.3677     
Epoch 58
1719/1719 [==============================] - 6s - loss: 0.0256 - acc: 0.3554     
Epoch 59
1719/1719 [==============================] - 6s - loss: 0.0256 - acc: 0.3589     
Epoch 60
1719/1719 [==============================] - 6s - loss: 0.0253 - acc: 0.3677     
Epoch 61
1719/1719 [==============================] - 6s - loss: 0.0255 - acc: 0.3514     
Epoch 62
1719/1719 [==============================] - 6s - loss: 0.0252 - acc: 0.4014     
Epoch 63
1719/1719 [==============================] - 6s - loss: 0.0251 - acc: 0.3950     
Epoch 64
1719/1719 [==============================] - 6s - loss: 0.0251 - acc: 0.3828     
Epoch 65
1719/1719 [==============================] - 6s - loss: 0.0250 - acc: 0.3909     
Epoch 66
1719/1719 [==============================] - 6s - loss: 0.0250 - acc: 0.3979     
Epoch 67
1719/1719 [==============================] - 6s - loss: 0.0251 - acc: 0.4002     
Epoch 68
1719/1719 [==============================] - 6s - loss: 0.0249 - acc: 0.3944     
Epoch 69
1719/1719 [==============================] - 7s - loss: 0.0248 - acc: 0.4078     
Epoch 70
1719/1719 [==============================] - 7s - loss: 0.0249 - acc: 0.4113     
Epoch 71
1719/1719 [==============================] - 8s - loss: 0.0247 - acc: 0.4113     
Epoch 72
1719/1719 [==============================] - 6s - loss: 0.0248 - acc: 0.4241     
Epoch 73
1719/1719 [==============================] - 6s - loss: 0.0246 - acc: 0.4194     
Epoch 74
1719/1719 [==============================] - 6s - loss: 0.0247 - acc: 0.3967     
Epoch 75
1719/1719 [==============================] - 6s - loss: 0.0244 - acc: 0.4171     
Epoch 76
1719/1719 [==============================] - 6s - loss: 0.0245 - acc: 0.4212     
Epoch 77
1719/1719 [==============================] - 6s - loss: 0.0245 - acc: 0.4200     
Epoch 78
1719/1719 [==============================] - 6s - loss: 0.0245 - acc: 0.4090     
Epoch 79
1719/1719 [==============================] - 6s - loss: 0.0245 - acc: 0.4206     
Epoch 80
1719/1719 [==============================] - 6s - loss: 0.0242 - acc: 0.4351     
Epoch 81
1719/1719 [==============================] - 6s - loss: 0.0242 - acc: 0.4549     
Epoch 82
1719/1719 [==============================] - 6s - loss: 0.0243 - acc: 0.4276     
Epoch 83
1719/1719 [==============================] - 6s - loss: 0.0246 - acc: 0.4101     
Epoch 84
1719/1719 [==============================] - 6s - loss: 0.0241 - acc: 0.4398     
Epoch 85
1719/1719 [==============================] - 6s - loss: 0.0240 - acc: 0.4351     
Epoch 86
1719/1719 [==============================] - 6s - loss: 0.0244 - acc: 0.3950     
Epoch 87
1719/1719 [==============================] - 6s - loss: 0.0241 - acc: 0.4439     
Epoch 88
1719/1719 [==============================] - 6s - loss: 0.0241 - acc: 0.4346     
Epoch 89
1719/1719 [==============================] - 6s - loss: 0.0240 - acc: 0.4683     
Epoch 90
1719/1719 [==============================] - 6s - loss: 0.0239 - acc: 0.4503     
Epoch 91
1719/1719 [==============================] - 6s - loss: 0.0238 - acc: 0.4729     
Epoch 92
1719/1719 [==============================] - 6s - loss: 0.0237 - acc: 0.4479     
Epoch 93
1719/1719 [==============================] - 6s - loss: 0.0237 - acc: 0.4741     
Epoch 94
1719/1719 [==============================] - 6s - loss: 0.0235 - acc: 0.5015     
Epoch 95
1719/1719 [==============================] - 6s - loss: 0.0238 - acc: 0.4561     
Epoch 96
1719/1719 [==============================] - 6s - loss: 0.0236 - acc: 0.4968     
Epoch 97
1719/1719 [==============================] - 6s - loss: 0.0237 - acc: 0.4764     
Epoch 98
1719/1719 [==============================] - 6s - loss: 0.0235 - acc: 0.4823     
Epoch 99
1719/1719 [==============================] - 6s - loss: 0.0239 - acc: 0.4410     
New loss
1719/1719 [==============================] - 1s     
[0.023333431469138934, 0.51018033740546831]

In [145]:

    

print(lstmmodel.evaluate(trainX,trainy,show_accuracy=True))


    

    




1719/1719 [==============================] - 1s     
[0.023333431469138934, 0.51018033740546831]

In [146]:

    

lstmyaml=lstmmodel.to_yaml()
print(lstmyaml)


    

    




class_mode: categorical
layers:
- {activation: tanh, forget_bias_init: one, init: glorot_uniform, inner_activation: hard_sigmoid,
  inner_init: orthogonal, input_dim: 33, name: LSTM, output_dim: 60, return_sequences: true,
  truncate_gradient: -1}
- {activation: tanh, forget_bias_init: one, init: glorot_uniform, inner_activation: hard_sigmoid,
  inner_init: orthogonal, input_dim: 60, name: LSTM, output_dim: 60, return_sequences: false,
  truncate_gradient: -1}
- {W_constraint: null, W_regularizer: null, activation: linear, activity_regularizer: null,
  b_constraint: null, b_regularizer: null, init: glorot_uniform, input_dim: 60, name: Dense,
  output_dim: 33}
loss: mean_squared_error
name: Sequential
optimizer: {decay: 0.0, lr: 0.1, momentum: 0.8, name: SGD, nesterov: false}
theano_mode: null

In [152]:

    

lstmcopy = model_from_yaml(lstmyaml)
print(lstmcopy.evaluate(trainX,trainy,show_accuracy=True))
lstmweights=lstmmodel.get_weights()
lstmcopy.set_weights(lstmweights)
print("Original:", lstmmodel.evaluate(trainX,trainy,show_accuracy=True))
print("Copy:",lstmcopy.evaluate(trainX,trainy,show_accuracy=True))


    

    




1719/1719 [==============================] - 1s     
[0.043840713632131052, 0.022687609075043629]
1719/1719 [==============================] - 1s     
Original: [0.023333431469138934, 0.51018033740546831]
1719/1719 [==============================] - 1s     
Copy: [0.023333431469138934, 0.51018033740546831]

In [153]:

    

jzs1model = Sequential()
jzs1model.add(JZS1(33,60, return_sequences=True))
jzs1model.add(JZS1(60,60))
jzs1model.add(Dense(60,33))
jzs1model.compile(optimizer=SGD(lr=0.1,momentum=0.8),loss='mse',class_mode="categorical")


    

    




DEBUG: nvcc STDOUT mod.cu
   Creating library D:/Users/nico/AppData/Local/Theano/compiledir_Windows-7-6.1.7601-SP1-AMD64_Family_21_Model_2_Stepping_0_AuthenticAMD-3.4.3-64/tmpopwc0glz/m2f8b298e743455e6e0d4423a9bb3d736.lib and object D:/Users/nico/AppData/Local/Theano/compiledir_Windows-7-6.1.7601-SP1-AMD64_Family_21_Model_2_Stepping_0_AuthenticAMD-3.4.3-64/tmpopwc0glz/m2f8b298e743455e6e0d4423a9bb3d736.exp

DEBUG: nvcc STDOUT mod.cu
   Creating library D:/Users/nico/AppData/Local/Theano/compiledir_Windows-7-6.1.7601-SP1-AMD64_Family_21_Model_2_Stepping_0_AuthenticAMD-3.4.3-64/tmppk2c_sim/me3622db1900040e92072ee3096913150.lib and object D:/Users/nico/AppData/Local/Theano/compiledir_Windows-7-6.1.7601-SP1-AMD64_Family_21_Model_2_Stepping_0_AuthenticAMD-3.4.3-64/tmppk2c_sim/me3622db1900040e92072ee3096913150.exp

DEBUG: nvcc STDOUT mod.cu
   Creating library D:/Users/nico/AppData/Local/Theano/compiledir_Windows-7-6.1.7601-SP1-AMD64_Family_21_Model_2_Stepping_0_AuthenticAMD-3.4.3-64/tmppjjej7gi/ma4e9f27cd8b6607a974149aa15e08351.lib and object D:/Users/nico/AppData/Local/Theano/compiledir_Windows-7-6.1.7601-SP1-AMD64_Family_21_Model_2_Stepping_0_AuthenticAMD-3.4.3-64/tmppjjej7gi/ma4e9f27cd8b6607a974149aa15e08351.exp

In [154]:

    

print("Initial loss")
print(jzs1model.evaluate(trainX,trainy,show_accuracy=True))
print("Training")
import time
start=time.time()
jzs1model.fit(trainX,trainy,nb_epoch = 5, show_accuracy=True,verbose=1)
print("Turning verbose off and training 100 epochs, this will take approx.", 20*(time.time()-start),"seconds")
from sys import stdout
stdout.flush()
jzs1model.fit(trainX,trainy,nb_epoch = 100, show_accuracy=True,verbose=0)
print("New loss")
print(jzs1model.evaluate(trainX,trainy,show_accuracy=True))


    

    




Initial loss
1719/1719 [==============================] - 1s     
[0.042056869431558633, 0.011634671320535195]
Training
Epoch 0
1719/1719 [==============================] - 3s - loss: 0.0392 - acc: 0.0145     
Epoch 1
1719/1719 [==============================] - 3s - loss: 0.0365 - acc: 0.0425     
Epoch 2
1719/1719 [==============================] - 3s - loss: 0.0356 - acc: 0.0762     
Epoch 3
1719/1719 [==============================] - 3s - loss: 0.0350 - acc: 0.0931     
Epoch 4
1719/1719 [==============================] - 3s - loss: 0.0344 - acc: 0.0989     
Turning verbose off and training 100 epochs, this will take approx. 398.0827760696411 seconds
New loss
1719/1719 [==============================] - 0s     
[0.023093559178390982, 0.56719022687609078]

In [157]:

    

jzs1yaml=jzs1model.to_yaml()
print(jzs1yaml)


    

    




class_mode: categorical
layers:
- {activation: tanh, init: glorot_uniform, inner_activation: sigmoid, inner_init: orthogonal,
  input_dim: 33, name: JZS1, output_dim: 60, return_sequences: true, truncate_gradient: -1}
- {activation: tanh, init: glorot_uniform, inner_activation: sigmoid, inner_init: orthogonal,
  input_dim: 60, name: JZS1, output_dim: 60, return_sequences: false, truncate_gradient: -1}
- {W_constraint: null, W_regularizer: null, activation: linear, activity_regularizer: null,
  b_constraint: null, b_regularizer: null, init: glorot_uniform, input_dim: 60, name: Dense,
  output_dim: 33}
loss: mean_squared_error
name: Sequential
optimizer: {decay: 0.0, lr: 0.1, momentum: 0.8, name: SGD, nesterov: false}
theano_mode: null

In [158]:

    

jzs1copy = model_from_yaml(jzs1yaml)
print("Untrained:",jzs1copy.evaluate(trainX,trainy,show_accuracy=True))
jzs1weights = jzs1model.get_weights()
jzs1copy.set_weights(jzs1weights)
print("Original:", jzs1model.evaluate(trainX,trainy,show_accuracy=True))
print("Copy:",jzs1copy.evaluate(trainX,trainy,show_accuracy=True))


    

    




1719/1719 [==============================] - 0s     
Untrained: [0.045702425345839147, 0.022687609075043629]
1719/1719 [==============================] - 0s     
Original: [0.023093559178390982, 0.56719022687609078]
1719/1719 [==============================] - 0s     
Copy: [0.040583413824603752, 0.056428155904595698]

In [ ]: